#!/usr/bin/python3
from spider.data_storage import DataStorage
from spider.html_downloader import HtmlDownloader
from spider.html_parser import HtmlParser
from spider.url_manager import UrlManager


class SpiderMain:
    def __init__(self):
        self.url_manager = UrlManager()
        self.html_downloader = HtmlDownloader()
        self.html_parser = HtmlParser()
        self.data_storage = DataStorage()

    def start(self):
        """
        开启爬虫
        @return:
        """
        # 添加url
        self.url_manager.add_new_url("http://127.0.0.1:8848/xiaomi-master/index.html")
        # 从url管理器中获取url
        url = self.url_manager.get_new_url()
        # 将获取到的url使用下载器下载
        html = self.html_downloader.download(url)
        # 将html进行解析
        result = self.html_parser.parser(html)
        # 数据存储
        self.data_storage.storage(result)


if __name__ == '__main__':

    main = SpiderMain()
    main.start()
